{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "#import socket\n",
    "#import socketserver\n",
    "\n",
    "from pynput.mouse import Button, Controller\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap=cv2.VideoCapture(0)#Camera capture of cv\n",
    "#Haar Cascade classifier for face and eyes\n",
    "face_cascade = cv2.CascadeClassifier('D:\\\\ProgramData\\\\Anaconda3\\\\Lib\\\\site-packages\\\\cv2\\\\data\\\\haarcascade_frontalface_default.xml')\n",
    "eye_cascade = cv2.CascadeClassifier('D:\\\\ProgramData\\\\Anaconda3\\\\Lib\\\\site-packages\\\\cv2\\\\data\\\\haarcascade_eye_tree_eyeglasses.xml')\n",
    "font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "\n",
    "'''\n",
    "host = '192.168.3.27'#host server ip\n",
    "port = 18888\n",
    "'''\n",
    "\n",
    "mouse = Controller()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nclass Myserver(socketserver.BaseRequestHandler):\\n    def handle(self):\\n        while(True):\\n            client = self.request\\n            print(\\'Client connected\\')\\n            #buf = client.recv(1024)\\n            try:\\n                client.send(\"11\")\\n            except:\\n                print(\\'socket.error\\')\\n                return\\nserver = socketserver.ThreadingTCPServer((host,port),Myserver)\\nserver.serve_forever()\\n'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "#Data Transmitting Module(In work)\n",
    "#Because Raspberry Pi cannot use SID directly to work as keyboard,I try to transmit data to computer so as to move mouse on computer.\n",
    "#In this way,Pi works as server and computer works as client,this .py file works as server program.\n",
    "#TCP protocal requires that both pi and computer are in the same LAN and their IP addresses shall be known.\n",
    "#Perhaps Bluetooth connection is a better solution.\n",
    "class Myserver(socketserver.BaseRequestHandler):\n",
    "    def handle(self):\n",
    "        while(True):\n",
    "            client = self.request\n",
    "            print('Client connected')\n",
    "            #buf = client.recv(1024)\n",
    "            try:\n",
    "                client.send(\"11\")\n",
    "            except:\n",
    "                print('socket.error')\n",
    "                return\n",
    "server = socketserver.ThreadingTCPServer((host,port),Myserver)\n",
    "server.serve_forever()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Enhanced data transmission module using Bluetooth\n",
    "import bluetooth\n",
    "target_name = 'LAPTOP-CEVGGQPF'\n",
    "target_address = None\n",
    "nearby_devices = bluetooth.discover_devices(lookup_names=True)\n",
    "for bdaddr in nearby_devices:\n",
    "    if target_name == bluetooth.lookup_name( bdaddr ):\n",
    "        target_address = bdaddr\n",
    "        break\n",
    "if target_address is not None:\n",
    "    print(\"found target bluetooth device with address \", target_address)\n",
    "else:\n",
    "    print(\"could not find target bluetooth device nearby\")\n",
    "    \n",
    "sock=bluetooth.BluetoothSocket( bluetooth.RFCOMM )\n",
    "try:\n",
    "    sock.connect((target_address,1))\n",
    "    print(\"connection successful\")\n",
    "except Exception as e:\n",
    "    print(\"connection fail\\n\",e)\n",
    "    sock.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "facePos = np.array([0,0,0,0])\n",
    "eyePos = np.array([[0,0,0,0],[10,0,0,0]])\n",
    "leyePos = np.array([0,0,0,0])\n",
    "staticPos = np.array([0,0,0,0])\n",
    "staticEyePos = np.array([0,0,0,0])\n",
    "isStatic = False\n",
    "settled = False\n",
    "mouseMov = False\n",
    "def sharpen(image):\n",
    "    kernel = np.array([[0,-1,0],[-1,4.7,-1],[0,-1,0]],np.float32)#Laplace算子实现滤波器\n",
    "    #Acturally using the matrix[[0,-1,0],[-1,5,-1],[0,-1,0]] works as sharpening and this one above breaks the image.\n",
    "    #But this kernal filter makes eyes and pupils more outstanding.\n",
    "    #The data of K22 indicates details.The smaller it is,the details are fewer.4.4~4.7 works well.\n",
    "    dst = cv2.filter2D(image,-1,kernel=kernel)\n",
    "    return dst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "while(cap.isOpened()):\n",
    "    ret,frame = cap.read()\n",
    "    #frame = cv2.imread('D:\\\\1.jpg')\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)\n",
    "    gray = cv2.equalizeHist(gray)\n",
    "    faces = face_cascade.detectMultiScale(gray, 1.3, 5)\n",
    "    #Detect face in gray frame\n",
    "    #Canny contour detecting includes four steps,but gaussian blur is not applied in the function cv2.Canny()\n",
    "    #So blurring is vital to prevent too many unnecessary details\n",
    "    shp = sharpen(frame)\n",
    "    shp = cv2.GaussianBlur(shp,(7,7),0)\n",
    "    gshp = cv2.cvtColor(shp, cv2.COLOR_BGR2GRAY)\n",
    "    canny = cv2.Canny(gshp,45,100)#The two parameters works as threshold,details between the two values are shown,bigger value means less details.\n",
    "    #Face & Eyes Detection Module \n",
    "    for (x,y,w,h) in faces:\n",
    "        frame = cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)#Showing face area\n",
    "        roi_gray = gray[y:int(y+h/2), x:x+w]\n",
    "        roi_color = frame[y:int(y+h/2), x:x+w]\n",
    "        eyes = eye_cascade.detectMultiScale(roi_gray)#Caution: the array is based on roi_gray, which is relative to faces[i].\n",
    "        #Because the array 'eyes' is not fixed in length, try/except is necessary in every process with 'eyes' or 'eyePos'.\n",
    "        #Perhaps this might be improved in further versions.\n",
    "        try:\n",
    "            if(len(eyes)>=1 or eyes!=None):\n",
    "                eyePos = list(eyes)\n",
    "            else:\n",
    "                eyePos = [[0,0,0,0],[10,0,0,0]]\n",
    "            if(eyes[0][0]<w/2):\n",
    "                leyePos = eyes[0]#Detect the coordinates of the left eye.Works not so well when head turns left.(Detecting right eye recommended)\n",
    "            facePos = faces[0]#This program is designed for single face detection.\n",
    "        except:\n",
    "            pass\n",
    "        for (ex,ey,ew,eh) in eyes:\n",
    "            cv2.circle(roi_color,(int(ex+ew/2),int(ey+eh/2)),2,(0,255,0),-1)#Point out pupil in green,acturally the center of eye area.\n",
    "            cv2.rectangle(roi_color,(ex,ey),(ex+ew,ey+eh),(0,255,0),2)#Shows eye area.\n",
    "            pass\n",
    "    \n",
    "    #Static state module\n",
    "    #Static state records the data of face and eyes of one time and freeze them in arrays:'staticPos' and 'staticEyePos'\n",
    "    #Static also works as a switch for mouse movement.Only when isStatic==Ture and staticEyePos[0][0] is in the left part of face area,\n",
    "    #that is to say,(isStatic and staticEyePos[0][0]<facePos[2]/2),can mouse be moved by this program.\n",
    "    i = 0\n",
    "    try:\n",
    "        if(isStatic==False):\n",
    "            settled = False#'settled' just makes sure that data won't be refreshed after isStatic\n",
    "        for (eex,eey,eew,eeh) in eyePos:\n",
    "            cv2.putText(frame,f'posX={eex},posY={eey},w={eew},h={eeh}',(10,300-i*30),font,0.8,(255,255,255),2)#This is pos of face and eyes in present \n",
    "            i=i+1\n",
    "    except:\n",
    "        pass\n",
    "    i=1\n",
    "    if(isStatic):\n",
    "        if(settled == False):\n",
    "            settled = True\n",
    "            staticEyePos = eyePos\n",
    "            staticPos = facePos\n",
    "        cv2.putText(frame,'Static',(10,330),font,0.8,(255,255,255),2)\n",
    "        for (eex,eey,eew,eeh) in staticEyePos:\n",
    "            cv2.putText(frame,f'Static Eye Pos(Rlt to Face),posX={eex},posY={eey},w={eew},h={eeh}',(10,330+i*30),font,0.8,(255,255,255),2)\n",
    "            i=i+1\n",
    "        cv2.putText(frame,f'Static Face Position,posX={staticPos[0]},posY={staticPos[1]}',(10,330+i*30),font,0.8,(255,255,255),2)\n",
    "    #Pupil detection & contour drawing Module(In work)\n",
    "    #This module aims at using Canny contour detection to find one's pupil.\n",
    "    #Now the 'canny' image can show one's pupil(actually works way better without glasses),but it hard to detect the index of contour.\n",
    "    #Finally the gravity center method will be applied to find pupil's coordinates,realizing pupil movement detection.\n",
    "    ret,thresh = cv2.threshold(canny,127,255,0)\n",
    "    contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)\n",
    "    ii=0\n",
    "    \n",
    "    count = [0,0]\n",
    "    grvsum = [[0,0],[0,0]]\n",
    "    #count[index of eye]\n",
    "    #grvsum[index of eye][the x or y] \n",
    "    \n",
    "    for index,contr in enumerate(contours):\n",
    "        M=cv2.moments(contr)\n",
    "        if(M['m00'] != 0):\n",
    "            cx=int(M['m10']/M['m00'])\n",
    "            cy=int(M['m01']/M['m00'])#Gravity center coordinates\n",
    "            x,y,w,h = cv2.boundingRect(contr)#Use boundingRect(contour) to show the contour's area.\n",
    "        else:\n",
    "            continue\n",
    "        for ind,po in enumerate(eyePos):#for most of the times,ind = 0 indicates left eye while ind = 1 indicates right eye\n",
    "            frame = cv2.rectangle(frame,(po[0]+facePos[0],po[1]+facePos[1]),(po[0]+po[2]+facePos[0],po[1]+po[3]+facePos[1]),(0,0,255),2)\n",
    "            if(x>po[0]+facePos[0]+14 and y>po[1]+facePos[1]+10 and x+w<po[0]+po[2]+facePos[0]-14 and y+h<po[1]+po[3]+facePos[1]-10):\n",
    "                frame = cv2.drawContours(frame,contours,index,(0,0,255),3)\n",
    "                cv2.putText(frame,f'Contour{index}:cx={cx},cy={cy}',(10,50+ind*20),font,0.6,(255,255,255),2)\n",
    "                if(ind<2):\n",
    "                    grvsum[ind][0]+=cx\n",
    "                    grvsum[ind][1]+=cy\n",
    "                    count[ind]+=1\n",
    "    #cv2.drawContours(frame,contours,-1,(0,0,255),3)\n",
    "    if(count[0] != 0 and count[1] != 0):\n",
    "        cv2.putText(frame,f'Contour center of grv avg:xl={int(grvsum[0][0]/count[0])},yl={int(grvsum[0][1]/count[1])},xr={int(grvsum[1][0]/count[0])},yr={int(grvsum[1][1]/count[1])}',(10,110),font,0.8,(255,255,255),2)\n",
    "\n",
    "    \n",
    "    cv2.imshow('frame',frame)\n",
    "    \n",
    "    \n",
    "    '''\n",
    "    try:\n",
    "        if(eyePos[0][0]<facePos[3]/2):\n",
    "            mouse.position = (eyePos[0][0]*5,eyePos[0][1]*5)\n",
    "    except:\n",
    "        pass\n",
    "    '''\n",
    "    \n",
    "    #Judgement module\n",
    "    #Only in use when isStatic and staticEyePos[0] shows the data of left eye\n",
    "    LOrR = 0\n",
    "    UpOrDown = 0\n",
    "    if(isStatic):\n",
    "        try:\n",
    "            if(staticEyePos[0][0]+staticEyePos[0][2]/2<staticPos[2]/2):\n",
    "                lpos = staticEyePos[0]\n",
    "            else:\n",
    "                break\n",
    "            if(facePos[0]+leyePos[0]>staticPos[0]+lpos[0]+20 and facePos[0]+leyePos[0]<staticPos[0]+lpos[0]+120):\n",
    "                #The eye moves leftward,so move the mouse to left\n",
    "                #take the speed in the following formula:v=Int((Abs(eyeX-staticX)-10)^0.6+15)\n",
    "                mouse.move(-int((facePos[0]+leyePos[0]-staticEyePos[0][0]-staticPos[0]-10)**0.6+15),0)\n",
    "                LOrR=1\n",
    "                pass\n",
    "            elif(facePos[0]+leyePos[0]<staticPos[0]+lpos[0]-20 and facePos[0]+leyePos[0]>staticPos[0]+lpos[0]-120):#moves right\n",
    "                mouse.move(int((staticPos[0]+staticEyePos[0][0]-facePos[0]-leyePos[0]-10)**0.6+15),0)\n",
    "                LOrR=-1\n",
    "                pass\n",
    "            else:\n",
    "                LOrR=0\n",
    "            if(facePos[1]+leyePos[1]>staticPos[1]+lpos[1]+12 and facePos[1]+leyePos[1]<staticPos[1]+lpos[1]+80):\n",
    "                mouse.move(0,int((facePos[1]+leyePos[1]-staticEyePos[0][1]-staticPos[1]-12)**0.6+15))\n",
    "                UpOrDown=1\n",
    "            elif(facePos[1]+leyePos[1]<staticPos[1]+lpos[1]-12 and facePos[1]+leyePos[1]>staticPos[1]+lpos[1]-120):\n",
    "                mouse.move(0,-int((staticPos[1]+staticEyePos[0][1]-facePos[1]-leyePos[1]-12)**0.6+15))\n",
    "                UpOrDown=-1\n",
    "            else:\n",
    "                UpOrDown=0\n",
    "            print(f'{LOrR},{UpOrDown}')\n",
    "            sock.send(f'{LOrR},{UpOrDown}')\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    \n",
    "    if cv2.waitKey(10) & 0xFF == ord('s'):\n",
    "        isStatic = bool(1-isStatic)\n",
    "    if cv2.waitKey(10) & 0xFF == ord('m'):\n",
    "        isStatic = bool(1-mouseMov)\n",
    "    if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
